{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "X = iris[\"data\"]# petal length, petal width\n",
    "y = iris[\"target\"].astype(np.float64) # Iris-Virginica\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('lr',\n",
       "                              LogisticRegression(C=1.0, class_weight=None,\n",
       "                                                 dual=False, fit_intercept=True,\n",
       "                                                 intercept_scaling=1,\n",
       "                                                 l1_ratio=None, max_iter=100,\n",
       "                                                 multi_class='warn',\n",
       "                                                 n_jobs=None, penalty='l2',\n",
       "                                                 random_state=None,\n",
       "                                                 solver='warn', tol=0.0001,\n",
       "                                                 verbose=0, warm_start=False)),\n",
       "                             ('rf',\n",
       "                              RandomForestClassifier(bootstrap=True,\n",
       "                                                     class_weight=None,\n",
       "                                                     criterion='gini',...\n",
       "                                                     oob_score=False,\n",
       "                                                     random_state=None,\n",
       "                                                     verbose=0,\n",
       "                                                     warm_start=False)),\n",
       "                             ('svc',\n",
       "                              SVC(C=1.0, cache_size=200, class_weight=None,\n",
       "                                  coef0=0.0, decision_function_shape='ovr',\n",
       "                                  degree=3, gamma='auto_deprecated',\n",
       "                                  kernel='rbf', max_iter=-1, probability=False,\n",
       "                                  random_state=None, shrinking=True, tol=0.001,\n",
       "                                  verbose=False))],\n",
       "                 flatten_transform=True, n_jobs=None, voting='hard',\n",
       "                 weights=None)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "rnd_clf = RandomForestClassifier()\n",
    "svm_clf = SVC()\n",
    "voting_clf = VotingClassifier(estimators=[('lr', log_clf), ('rf', rnd_clf),\n",
    "                                          ('svc', svm_clf)], voting='hard')\n",
    "voting_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression 0.9666666666666667\n",
      "RandomForestClassifier 0.9333333333333333\n",
      "SVC 0.9666666666666667\n",
      "VotingClassifier 0.9666666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "bag_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500,\n",
    "                           max_samples=100, bootstrap=True, n_jobs=-1)\n",
    "bag_clf.fit(X_train, y_train)\n",
    "y_pred = bag_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, \n",
    "                            bootstrap=True, n_jobs=-1, oob_score=True)\n",
    "bag_clf.fit(X_train, y_train)\n",
    "bag_clf.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9666666666666667"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = bag_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.70658683, 0.29341317],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.83870968, 0.16129032],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.01086957, 0.98913043],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.15306122, 0.84693878],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.046875  , 0.953125  ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.00546448, 0.99453552],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.91133005, 0.08866995],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.9939759 , 0.0060241 ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.01123596, 0.98876404],\n",
       "       [0.        , 0.02061856, 0.97938144],\n",
       "       [0.        , 0.84153005, 0.15846995],\n",
       "       [0.        , 0.20903955, 0.79096045],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.89784946, 0.10215054],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.00564972, 0.99435028],\n",
       "       [0.        , 0.94674556, 0.05325444],\n",
       "       [0.        , 0.00564972, 0.99435028],\n",
       "       [0.        , 0.01104972, 0.98895028],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.84491979, 0.15508021],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.00510204, 0.99489796],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.00540541, 0.99459459],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.2       , 0.8       ],\n",
       "       [0.        , 0.85635359, 0.14364641],\n",
       "       [0.        , 0.98930481, 0.01069519],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.01123596, 0.98876404],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.41621622, 0.58378378],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.95652174, 0.04347826],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.02259887, 0.97740113],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.97927461, 0.02072539],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.95428571, 0.04571429],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.41752577, 0.58247423],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.01030928, 0.98969072],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.92134831, 0.07865169],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [0.        , 0.96842105, 0.03157895],\n",
       "       [0.        , 0.        , 1.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.02824859, 0.97175141],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 1.        , 0.        ],\n",
       "       [1.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 1.        ]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf.oob_decision_function_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1)\n",
    "rnd_clf.fit(X_train, y_train)\n",
    "y_pred_rf = rnd_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_clf = BaggingClassifier(DecisionTreeClassifier(splitter=\"random\", max_leaf_nodes=16)\n",
    "                           ,n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sepal length (cm) 0.09971244897625198\n",
      "sepal width (cm) 0.02456383207605974\n",
      "petal length (cm) 0.45692877863855164\n",
      "petal width (cm) 0.4187949403091367\n"
     ]
    }
   ],
   "source": [
    "rnd_clf = RandomForestClassifier(n_estimators=500, n_jobs=-1)\n",
    "rnd_clf.fit(iris[\"data\"], iris[\"target\"])\n",
    "for name, score in zip(iris[\"feature_names\"], rnd_clf.feature_importances_):\n",
    "    print(name, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "                   base_estimator=DecisionTreeClassifier(class_weight=None,\n",
       "                                                         criterion='gini',\n",
       "                                                         max_depth=1,\n",
       "                                                         max_features=None,\n",
       "                                                         max_leaf_nodes=None,\n",
       "                                                         min_impurity_decrease=0.0,\n",
       "                                                         min_impurity_split=None,\n",
       "                                                         min_samples_leaf=1,\n",
       "                                                         min_samples_split=2,\n",
       "                                                         min_weight_fraction_leaf=0.0,\n",
       "                                                         presort=False,\n",
       "                                                         random_state=None,\n",
       "                                                         splitter='best'),\n",
       "                   learning_rate=0.5, n_estimators=200, random_state=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), n_estimators=200,\n",
    "                            algorithm=\"SAMME.R\", learning_rate=0.5)\n",
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "                      max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      presort=False, random_state=None, splitter='best')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "tree_reg1 = DecisionTreeRegressor(max_depth=2)\n",
    "tree_reg1.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "                      max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      presort=False, random_state=None, splitter='best')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2 = y - tree_reg1.predict(X)\n",
    "tree_reg2 = DecisionTreeRegressor(max_depth=2)\n",
    "tree_reg2.fit(X, y2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "                      max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      presort=False, random_state=None, splitter='best')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y3 = y2 - tree_reg2.predict(X)\n",
    "tree_reg3 = DecisionTreeRegressor(max_depth=2)\n",
    "tree_reg3.fit(X, y3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_new' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-e9801bfe54ea>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_new\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtree\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtree_reg1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtree_reg2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtree_reg3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-15-e9801bfe54ea>\u001b[0m in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_new\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtree\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtree_reg1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtree_reg2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtree_reg3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'X_new' is not defined"
     ]
    }
   ],
   "source": [
    "y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "                          learning_rate=1.0, loss='ls', max_depth=2,\n",
       "                          max_features=None, max_leaf_nodes=None,\n",
       "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                          min_samples_leaf=1, min_samples_split=2,\n",
       "                          min_weight_fraction_leaf=0.0, n_estimators=3,\n",
       "                          n_iter_no_change=None, presort='auto',\n",
       "                          random_state=None, subsample=1.0, tol=0.0001,\n",
       "                          validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0)\n",
    "gbrt.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "                          learning_rate=0.1, loss='ls', max_depth=2,\n",
       "                          max_features=None, max_leaf_nodes=None,\n",
       "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                          min_samples_leaf=1, min_samples_split=2,\n",
       "                          min_weight_fraction_leaf=0.0, n_estimators=29,\n",
       "                          n_iter_no_change=None, presort='auto',\n",
       "                          random_state=None, subsample=1.0, tol=0.0001,\n",
       "                          validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y)\n",
    "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120)\n",
    "gbrt.fit(X_train, y_train)\n",
    "errors = [mean_squared_error(y_val, y_pred)\n",
    "                           for y_pred in gbrt.staged_predict(X_val)]\n",
    "bst_n_estimators = np.argmin(errors)\n",
    "gbrt_best = GradientBoostingRegressor(max_depth=2, n_estimators=bst_n_estimators)\n",
    "gbrt_best.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True)\n",
    "min_val_error = float(\"inf\")\n",
    "error_going_up = 0\n",
    "for n_estimators in range(1, 120):\n",
    "    gbrt.n_estimators = n_estimators\n",
    "    gbrt.fit(X_train, y_train)\n",
    "    y_pred = gbrt.predict(X_val)\n",
    "    val_error = mean_squared_error(y_val, y_pred)\n",
    "    if val_error < min_val_error:\n",
    "        min_val_error = val_error\n",
    "        error_going_up = 0\n",
    "    else:\n",
    "        error_going_up += 1\n",
    "        if error_going_up == 5:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
